1、采集方案

Spring通过logback产生日志, 然后通过logback的一个TCP Appender产生结构化的Json日志数据, 并直接将该日志数据发送到Logstash监听的TCP端口, Logstash收到后再发送到Elasticsearch

2、实战

2.1、SpringBoot和Logback配置

2.1.1 核心依赖

implementation "net.logstash.logback:logstash-logback-encoder:6.2"

compileOnly "org.projectlombok:lombok"
annotationProcessor "org.projectlombok:lombok"

2.1.2 application.yml

spring:
  profiles:
    active: ${SPRING_ACTIVE:dev}

  application:
    name: elk-server

custom:
  logstash_uri: 127.0.0.1:4660
  appender_ref: Tcp_Logstash

2.1.3 logback-spring.xml

• Logstash appender的详细配置说明见此链接

<configuration>
    
    <conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
    <conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
    <conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>

    
    <property name="CONSOLE_LOG_PATTERN"
              value="[%clr(%X{traceId:-}){yellow}] %clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}"/>

    
    <springProperty scope="local" name="App_Name" source="spring.application.name"
                    defaultValue="demo-server"/>
    <springProperty scope="local" name="Appender_Ref" source="custom.appender_ref"
                    defaultValue="Console"/>
    <springProperty scope="local" name="Logstash_Uri" source="custom.logstash_uri"/>

    
    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${CONSOLE_LOG_PATTERN}pattern>
            <charset>UTF-8charset>
        encoder>
    appender>

 
    
     <appender name="Tcp_Logstash" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
         <keepAliveDuration>5 minuteskeepAliveDuration>
         
         <destination>${Logstash_Uri}destination>
         
         <encoder  charset="UTF-8" class="net.logstash.logback.encoder.LogstashEncoder">
             
             <timestampPattern>yyyy-MM-dd HH:mm:ss.SSStimestampPattern>
             
             <fieldNames>
                 <timestamp>timetimestamp>
                 <message>msgmessage>
                 <logger>classlogger>
                 
                 <version>[ignore]version>
                 <thread>[ignore]thread>
                 <levelValue>[ignore]levelValue>
             fieldNames>
             
             <includeMdcKeyName>traceIdincludeMdcKeyName>
             
             <customFields>{"appName": "${App_Name}"}customFields>
             
             <shortenedLoggerNameLength>40shortenedLoggerNameLength>
         encoder>
     appender>

    
    <logger name="org.slf4j" level="INFO"/>
    <logger name="springfox" level="INFO"/>
    <logger name="io.swagger" level="INFO"/>
    <logger name="org.springframework" level="INFO"/>
    <logger name="org.hibernate.validator" level="INFO"/>

    
    <springProfile name="dev">
        <root level="INFO">
            <appender-ref ref="Console" />
        root>
    springProfile>

    
    <springProfile name="prod">
        <root level="INFO">
            <appender-ref ref="${Appender_Ref}" />
            <appender-ref ref="Console"/>
        root>
    springProfile>

configuration>

2.2 Logstash 配置

1、创建一个conf文件: vim tcp_to_es.conf

// 监听一个4660的TCP端口接收数据
input {
  tcp {
    host => "localhost"
    port => 4660
    mode => "server"
    tags => ["burukeyou"]
    codec => json_lines
  }
}

output {
  // 输出到ES
  elasticsearch {
    // 输出的es地址
    hosts => ["http://localhost:9200"]
    // 输出到的索引, 如果不存在会自动创建名为"'appName'-'当日日期'"的索引
    index => "log-%{[appName]}-%{+YYYY.MM.dd}"
    #user => "elastic"
    #password => "changeme"
  }
  
  // 以rubydebug格式输出到Logstash的控制台
  stdout { codec => rubydebug }
}

2、启动logstash

• 启动后会默认再ES中创建一个 logstash-* 的索引模版

bin/logstash -f ../config/tcp_to_es.conf

3、启动SpringBoot并打印日志

@Slf4j
@Component
public class LogCreateListener implements ApplicationListener<ApplicationStartedEvent> {

    @Override
    public void onApplicationEvent(ApplicationStartedEvent event) {
        for (int i = 0; i <10; i++) {
            log.info("日志信息: {}", i);
        }
    }
}

2、Kibana 观察日志

2.1 创建索引模式

设置索引模式名

设置哪个字段为时间字段, 方便根据时间去观察日志

点击 Discover并选择索引模式为log-elk-server-* 就可以看到在这个索引模式下包含的所有索引日志数据

3、Kibana 可视化

1、点击 Visualize Library列表, 点击创建可视化

2、 选择要创建的图

3、配置图

• 配置后点机save保存即可